"""

.. module:: boot
.. moduleauthor:: @author: 

The purpose of this module is to control the node installation and boot. Note
that it is a goal of this module that it NOT include any other module except
for Python modules. This constraint allows boot.py to be standalone. However
it does have the overhead of requiring some code duplication with other modules
that would ordinarily be imported into this.
   
.. seealso:: 

*Constants*:
.. data:: DISTRIBUTION_DATASTORE_FILENAME_DEFAULT
        
    DISTRIBUTION_DATASTORE_FILENAME_DEFAULT is the default name of the distribution datastore.
    
.. data:: DOWNLOAD_SITE_URL_DEFAULT
        
    DOWNLOAD_SITE_URL_DEFAULT is the default URL of the download site.

.. data:: DOWNLOAD_SITE_DISTRIBUTION_DATASTORE_URL_DEFAULT
        
    DOWNLOAD_SITE_DISTRIBUTION_DATASTORE_URL_DEFAULT is the default URL of the distribution datastore.
    
.. data:: CENTRAL_NODE_URL_TEST
        
    CENTRAL_NODE_URL_TEST is the default URL of the central node (test).

.. data:: CENTRAL_NODE_URL_PRODUCTION
        
    CENTRAL_NODE_URL_PRODUCTION is the default URL of the central node (production).

.. data:: DEFAULT_PROXY_PORT
        
    DEFAULT_PROXY_PORT is the default port number of a proxy server.

.. data:: TEST_SITE_URL
        
    TEST_SITE_URL is the URL we request to test connectivity through a proxy.

.. data:: RESOURCE_KEY_URI_PACKAGE_PACKAGENAME
        
    RESOURCE_KEY_URI_PACKAGE_PACKAGENAME is the Resource name of a python package within the datastore.

.. data:: RESOURCE_KEY_URI_PACKAGE_MODULENAME
        
    RESOURCE_KEY_URI_PACKAGE_MODULENAME is the Resource name of a python module within the datastore.

.. data:: RESOURCE_ID
        
    RESOURCE_ID is used as an alias for the datastore's RESOURCE table 'id' field.

.. data:: RESOURCE_OBJECT
        
    RESOURCE_OBJECT is used as an alias for the datastore's RESOURCE table 'object' field.

.. data:: RESOURCE_METADATA_KEY
        
    RESOURCE_METADATA_KEY is used as an alias for the datastore's RESOURCEMETADATA table 'key' field.
"""

# Imports
#
import base64
import imp
import logging
from optparse import OptionParser
import os
# import signal
# import socket
import SocketServer
import sqlite3
import string
import sys
import threading
import urllib
import urllib2
import urlparse
import uuid
import xml.etree.ElementTree

if sys.platform is 'win32':
    import win32evtlog
    from logging.handlers import NTEventLogHandler
elif sys.platform is 'darwin': # OS/X
    pass
else: # assume Linux
    pass

# Constants
#
DISTRIBUTION_DATASTORE_FILENAME_DEFAULT = "temp.datastore"
DOWNLOAD_SITE_URL_DEFAULT = "http://yarra.googlecode.com/files"
DOWNLOAD_SITE_DISTRIBUTION_DATASTORE_URL_DEFAULT = ''.join([DOWNLOAD_SITE_URL_DEFAULT, "/", DISTRIBUTION_DATASTORE_FILENAME_DEFAULT])
CENTRAL_NODE_URL_TEST = "localhost"
CENTRAL_NODE_URL_PRODUCTION = "localhost"
DEFAULT_PROXY_PORT = 80
TEST_SITE_URL = "http://www.google.com"

# These are duplicated from datastore.resource.Resource because we don't want to import Resource.
#
RESOURCE_KEY_URI_PACKAGE_PACKAGENAME = "package:packagename"
RESOURCE_KEY_URI_PACKAGE_MODULENAME = "package:modulename"

# Database fields
#
RESOURCE_ID = 'RESOURCE_ID'
RESOURCE_OBJECT = 'RESOURCE_OBJECT'     
RESOURCE_METADATA_KEY = 'RESOURCE_METADATA_KEY'

# Classes
#
class Console(object):
    """The purpose of this class is to handle I/O from the console.

    *Usage examples*::

        example code

    *Common Arguments for Class methods*:
        <arguments>
    
    *Audience*:
        <audience>
    
    *Constants*:
        .. data:: constant_1
        
            constant_1 is used for ......
        
        .. data:: constant_2
        
            constant_2 is used for ......
    """
    
    # Class constants
    #
    
    def __init__(self):
        # Initialise the Console class.
        #
        # Parameters:
        #     <param 1> :
        #     ...
        #
        
        pass
    
    @staticmethod
    def display(msg):
        """This function displays the argument message on the console.
        
        :param msg: String - message to display to user.
        """
        
        print msg
    
    @staticmethod
    def getString(msg, default="", prompt="> ", mandatory=True):
        """This function displays the argument message on the console,
        displays a prompt on the console, waits for user input, and 
        returns the input, or 'default' if no input. A String value 
        is always returned. If 'mandatory' argument is True (default) 
        then user will be re-prompted if no value entered. However, if
        'mandatory' is True and 'default' is supplied then 'default
        will be returned if user enters no value.
        
        :param msg: String - message to display to user.
        :param default: String - default value to display to user. This 
        is returned if there is no input from the user.
        :param prompt: String - prompt to display to user awaiting input.
        :param mandatory: Boolean - Set True to re-prompt user if no value 
        entered.
        :rtype: String - the user input or default value, if any. A String, be
        it empty or not, is always returned.
        """
        
        input_string = ""
        if (default != None) and (len(default) > 0):
            input_string = ''.join([msg, ' (', default, ') ', prompt])
        else:
            input_string = ''.join([msg, ' ', prompt])

        gotInput = False
        while (gotInput is False):
            # strip leading and trailing whitespace from input. Assign 'default'
            # if no input returned.
            # 
            input = raw_input(input_string).strip()
            if (input == None) or (len(input) == 0):
                input = default
            
            gotInput = (len(input) > 0)
        
        return input

    @staticmethod
    def getInt(msg, default=0, prompt="> ", mandatory=True):
        """This function displays the argument message on the console,
        displays a prompt on the console, waits for user input, and 
        returns the input, or 'default' if no input. An integer value
        is always returned. If 'mandatory' argument is True (default) 
        then user will be re-prompted if no value entered. However, if
        'mandatory' is True and 'default' is supplied then 'default
        will be returned if user enters no value.
        
        :param msg: String - message to display to user.
        :param default: integer - default value to display to user. This 
        is returned if there is no input from the user.
        :param prompt: String - prompt to display to user awaiting input.
        :param mandatory: Boolean - Set True to re-prompt user if no value 
        entered.
        :rtype: integer - the user input or default value, if any. An integer 
        is always returned.
        """
        
        is_valid = False
        while (is_valid is False):
            try:
                val = 0
                if (default > 0):
                    val = int(Console.getString(msg, default=`default`, prompt=prompt, mandatory=mandatory))
                else:
                    val = int(Console.getString(msg, prompt=prompt, mandatory=mandatory))
                is_valid = True
            except ValueError:
                Console.display("Bad numeric value '%s'" % val)
        
        return val
    
    @staticmethod
    def getUrl(msg, default="", prompt="> ", mandatory=True, scheme="http"):
        """This function displays the argument message on the console,
        displays a prompt on the console, waits for user input, and 
        returns the input, or 'default' if no input. A tuple value
        is always returned. If 'mandatory' argument is True (default) 
        then user will be re-prompted if no value entered. However, if
        'mandatory' is True and 'default' is supplied then 'default
        will be returned if user enters no value. Any entered value (but
        not 'default') will be checked for URL validity (ie. a non-Null
        scheme and hostname).
        
        :param msg: String - message to display to user.
        :param default: integer - default value to display to user. This 
        is returned if there is no input from the user.
        :param prompt: String - prompt to display to user awaiting input.
        :param mandatory: Boolean - Set True to re-prompt user if no value 
        entered.
        :param scheme: String - If supplied, the input URL scheme must match 
        this argument.
        :rtype: tuple - the user input or default value, if any. A tuple
        is always returned.
        """
        
        urlTuple = tuple()
        is_valid = False
        while (is_valid is False):
            urlString = Console.getString(msg, default=default, prompt=prompt, mandatory=mandatory)
            urlTuple = urlparse.urlparse(urlString)
            
            # make sure input scheme matches argument scheme, but only if argument supplied.
            #
            if (len(scheme) > 0):
                if ((len(urlTuple.scheme) == 0) or (urlTuple.scheme.lower() != scheme.lower())):
                    Console.display("Bad URL scheme. Must be '%s'" % scheme)
                elif ((urlTuple.netloc == None) or (len(urlTuple.netloc) == 0)):
                    Console.display("Bad URL hostname.")
                else:
                    is_valid = True
            else:
                is_valid = True

        return urlTuple
    
    @staticmethod
    def getUrlAsString(msg, default="", prompt="> ", mandatory=True, scheme="http"):
        """This function displays the argument message on the console,
        displays a prompt on the console, waits for user input, and 
        returns the input, or 'default' if no input. A String value
        is always returned. If 'mandatory' argument is True (default) 
        then user will be re-prompted if no value entered. However, if
        'mandatory' is True and 'default' is supplied then 'default
        will be returned if user enters no value. Any entered value (but
        not 'default') will be checked for URL validity (ie. a non-Null
        scheme and hostname).
        
        :param msg: String - message to display to user.
        :param default: integer - default value to display to user. This 
        is returned if there is no input from the user.
        :param prompt: String - prompt to display to user awaiting input.
        :param mandatory: Boolean - Set True to re-prompt user if no value 
        entered.
        :param scheme: String - If supplied, the input URL scheme must match 
        this argument.
        :rtype: String - the user input or default value, if any. A String
        is always returned.
        """
        
        urlTuple = Console.getUrl(msg, default, prompt, mandatory, scheme)
        return urlTuple.geturl()
    

class Configurables(object):
    """The purpose of this class is to control the node's configurable items 
    during installation of the node software.

    *Usage examples*::

        example code

    *Common Arguments for Class methods*:
        <arguments>
    
    *Audience*:
        <audience>
    
    *Constants*:
        .. data:: constant_1
        
            constant_1 is used for ......
    """
    
    # Class constants
    #
    
    def __init__(self):
        # Initialise the Install class.
        #
        # Parameters:
        #     <param 1> :
        #     ...
        #
        
        self.proxy_url = tuple()
        self.proxy_user = ""
        self.proxy_password = ""
        self.firewall_download_site_url = tuple()
        self.firewall_admin_site_url = tuple()
        self.distribution_datastore_url = tuple()
        self.admin_site_url = CENTRAL_NODE_URL_PRODUCTION
        # self.node_url = ""
        self.node_ip_address = ""
        self.node_port_range_high = 0
        self.node_port_range_low = 0
        self.is_service = False
        self.using_proxy = False


class Distribution(object):
    """The purpose of this class is to control the download and unpacking of
    the distribution. 

    *Usage examples*::

        example code

    *Common Arguments for Class methods*:
        <arguments>
    
    *Audience*:
        <audience>
    
    *Constants*:
        .. data:: constant_1
        
            constant_1 is used for ......
    """
    
    # Class constants
    #
    
    def __init__(self):
        # Initialise the Install class.
        #
        # Parameters:
        #     <param 1> :
        #     ...
        #
        
        # self.distribution_xml = ""
        # self.distribution_xml_root = None
        self.exe_pathname = ""
        self.distribution_datastore_contents = None
        self.database_pathname = ""
        self.database_filename = ""
        self.node_ID = None    


class LoggingLevels(object):
    """The purpose of this class is to define extra logging levels.

    *Usage examples*::

        example code

    *Common Arguments for Class methods*:
        <arguments>
    
    *Audience*:
        <audience>
    
    *Constants*:
        .. data:: constant_1
        
            constant_1 is used for ......
        
        .. data:: constant_2
        
            constant_2 is used for ......
    """
    
    # Class constants
    #
    DEVEL = logging.INFO - 1            # Data to support a developer writing a new task.
    TRACE = logging.DEBUG - 1


class TCPServerHandler(SocketServer.BaseRequestHandler):
    """
    The RequestHandler class for our server.

    It is instantiated once per connection to the server, and must
    override the handle() method to implement communication to the
    client.
    """

    def __init__(self):
        # Initialise the NodeServer class.
        #
        # Parameters:
        #     <param 1> :
        #     ...
        #
        
        self.logger = logging.getLogger('yarra.runtime.boot.TCPServerHandler')
        
    def handle(self):
        self.logger.log(logging.DEBUG, "")

        # We're only interested in processing the connection request, not
        # data on the connection.
        # 

class NodeServer(threading.Thread):
    """The purpose of this class is to act as a temporary server, accepting 
    connections on the configured IP address and port number. This is to test
    connectivity to configured node IP address and port number range. 

    *Usage examples*::

        example code

    *Common Arguments for Class methods*:
        <arguments>
    
    *Audience*:
        <audience>
    
    *Constants*:
        .. data:: constant_1
        
            constant_1 is used for ......
    """
    
    # Class constants
    #
    
    def __init__(self, ip_address, port_number):
        # Initialise the NodeServer class.
        #
        # Parameters:
        #     <param 1> :
        #     ...
        #
        
        threading.Thread.__init__(self)
        
        self.logger = logging.getLogger('yarra.runtime.boot.NodeServer')
        self.ip_address = ip_address
        self.port_number = port_number
        
        try:
            self.server = SocketServer.TCPServer((self.ip_address, self.port_number), TCPServerHandler)
        except BaseException as be:
            self.logger.exception("Cannot bind to IP address/port number (%s:%s)" % (self.ip_address, self.port_number))
            raise Exception(be)
        
    def run(self):
        """This function is the mainline of the thread. It is called from start(). 
        """
        
        self.logger.log(logging.DEBUG, "Listening on (%s:%s)" % (self.ip_address, self.port_number))
        
        self.server.handle_request()


class Install(object):
    """The purpose of this class is to control the installation of the node software.

    *Usage examples*::

        example code

    *Common Arguments for Class methods*:
        <arguments>
    
    *Audience*:
        <audience>
    
    *Constants*:
        .. data:: constant_1
        
            constant_1 is used for ......
    """
    
    # Class constants
    #
    
    def __init__(self):
        # Initialise the Install class.
        #
        # Parameters:
        #     <param 1> :
        #     ...
        #
        
        self.logger = logging.getLogger('yarra.runtime.boot.Install')
    
    @staticmethod
    def download_url(download_url):
        """This function downloads the contents of the argument URL. 
        
        :param download_url: String - URL of file to download.
        :rtype: String - contents of the argument URL.
        """

        logging.log(logging.INFO, "Downloading URL (%s)" % download_url) 
        
        try:
            contents = None
            urlHandle = urllib.urlopen(download_url)
        except IOError:
            logging.exception("Connection failure to (%s)" % download_url) 
            raise IOError("download_url(): Connection failure to (%s)" % download_url)
        else:
            contents = urlHandle.read()
            urlHandle.close()
            urlHandle = None
        
        return contents

    @staticmethod
    def get_first_item_of_serialised_resource(serialised_resource):
        """This function returns the first item in the argument serialised resource. 
        
        :param serialised_resource: String - URL of file to download.
        :rtype: String - contents of the argument URL.
        """
        
        serialised_resource_xml_root = xml.etree.ElementTree.XML(serialised_resource)
        try:
            item = serialised_resource_xml_root.find('./content/item').text
        except AttributeError:
            logging.exception("Unexpected XML Structure")
            raise XMLFormatException("get_first_item_of_serialised_resource(): Unexpected XML Structure")
        
        return item
    
    @staticmethod
    def create_python_module(module_name):
        """This function creates a new python module object for the argument 
        module_name, if the module_name does not already exist. If it does 
        already exist then it is returned.
        
        :param module_name: String - name of module.
        :rtype: module object
        """
        
        logging.log(logging.INFO, "creating module (%s)..." % module_name)  
        
        if not module_name in sys.modules:
            # create a new module
            #
            module = imp.new_module(module_name)
            
            # add the module to the sys.modules as it will be used by other modules 
            #   
            sys.modules[module_name] = module
        else:
            module = sys.modules[module_name]
            
        return module

    @staticmethod
    def generate_node_ID(distribution):
        """This function generates a new node_ID - a UUID Type 4 (Random) number. 
        
        :param distribution: Distribution - distribution configuration.
        :rtype: None
        """
        
        distribution.node_ID = uuid.uuid4()
        logging.log(logging.DEBUG, "new node_ID (%s)" % distribution.node_ID)
        
    @staticmethod
    def get_network_configurables(configurables, distribution):
        """This function prompts the user for network configuration items. Those 
        items that describe communication end-points will have their connectivity
        checked. If connectivity fails the user will be re-prompted.
        
        :param configurables: Configurables - user configuration items.
        :param distribution: Distribution - distribution configuration.
        :rtype: None
        """

        # External access (1=direct, 2=proxy, 3=firewall port forward)
        #
        external_access = 0
        while (external_access < 1) or (external_access > 3):
            external_access = Console.getInt(msg="Enter external access (1 = direct, 2 = proxy, 3 = firewall port forward)", default=1, mandatory=True)
        
        if (external_access == 2):
            # get Proxy URL (mandatory) and credentials (optional)
            #
            urlHandle = None
            while (not urlHandle):
                configurables.proxy_url = Console.getUrl(msg="Enter Proxy URL", mandatory=True, scheme="http")
                configurables.proxy_user = Console.getString(msg="Enter Proxy user", mandatory=False)
                configurables.proxy_password = Console.getString(msg="Enter Proxy password", mandatory=False)
                
                try:
                    url_parts = string.split(configurables.proxy_url.netloc, ":")
                    if (len(url_parts) == 1):
                        url_parts.append(`DEFAULT_PROXY_PORT`)
                    
                    proxy_info = {
                        'user' : configurables.proxy_user,
                        'pass' : configurables.proxy_password,
                        'host' : url_parts[0],
                        'port' : url_parts[1]
                    }
                    
                    logging.log(logging.DEBUG, "Proxy details (%s)" % proxy_info) 
                    
                    proxy_support = urllib2.ProxyHandler({"http" : "http://%(user)s:%(pass)s@%(host)s:%(port)s" % proxy_info})
                    opener = urllib2.build_opener(proxy_support, urllib2.HTTPHandler)
                    urllib2.install_opener(opener)
                    
                    # download distribution file, just to test proxy connection.
                    #
                    urlHandle = urllib2.urlopen(TEST_SITE_URL)
                    
                except IOError:
                    Console.display("Connection failure through %s" % configurables.proxy_url.geturl())
                else:
                    configurables.using_proxy = True
                    
        elif (external_access == 3):
            # get firewall port-forward URL for download site. 
            #
            str = ""
            while (str == ""):
                configurables.firewall_download_site_url = Console.getUrl(msg="Enter firewall URL (to access download site)", mandatory=True, scheme="http") 
                try:
                    # Here we open a connection to the download site home page (ie. GET /) to prove we can connect through the firewall.
                    #
                    str = Install.download_url(configurables.firewall_download_site_url.geturl())
                except IOError:
                    Console.display("Connection failure to %s" % configurables.firewall_download_site_url.geturl())
            
            # get firewall port-forward URL of admin site.
            #
            str = ""
            while (str == ""):
                configurables.firewall_admin_site_url = Console.getUrl(msg="Enter firewall URL (to access admin site)", mandatory=True, scheme="http")
                try:
                    # Here we open a connection to the admin site home page (ie. GET /) to prove we can connect through the firewall.
                    #
                    str = Install.download_url(configurables.firewall_admin_site_url.geturl())
                except IOError:
                    Console.display("Connection failure to %s" % configurables.firewall_admin_site_url.geturl())
            
            # TODO: The format of the download and admin site URLs for port-forwarding will (probably) contain the firewall
            # IP address and a firewall port number. The firewall will map requests to an external IP address/port. Should
            # we therefore also capture the hostname for each of the download and admin site. Its likely that these sites
            # will need the hostname in the 'Host:' HTTP header field.
            # 
        
        # get URL for distribution datastore. If the user has entered port-forwarding URLs then this
        # URL must include the host:port of the firewall that is mapped to the host:port of the
        # download site.
        #
        # Allow user to load from current machine. Therefore don't restrict user to specific scheme.
        # (ie. allow 'http:', 'ftp:', 'file:', etc
        #
        while (distribution.distribution_datastore_contents == None):
            configurables.distribution_datastore_url = Console.getUrl(msg="Enter distribution datastore URL", default=DOWNLOAD_SITE_DISTRIBUTION_DATASTORE_URL_DEFAULT, mandatory=True, scheme="")
            try:
                distribution.distribution_datastore_contents = Install.download_url(configurables.distribution_datastore_url.geturl())
            except IOError:
                Console.display("Connection failure to %s" % configurables.distribution_datastore_url)
        
        # get URL of admin site. If the user has entered port-forwarding URLs then this
        # URL must include the host:port of the firewall that is mapped to the host:port of the
        # admin site.
        #
        str = ""
        while (str == ""):
            configurables.admin_site_url = Console.getUrlAsString(msg="Enter admin site URL", mandatory=True, scheme="http")
            try:
                str = Install.download_url(configurables.admin_site_url)
            except IOError:
                Console.display("Connection failure to %s" % configurables.admin_site_url)
        
        # get the node IP address. 
        #
        is_valid = False
        while (is_valid is False):
            configurables.node_ip_address = Console.getString(msg="Enter node IP address", mandatory=True)
            configurables.node_port_range_low = Console.getInt(msg="Enter node(s) port number range start", mandatory=True)
            configurables.node_port_range_high = Console.getInt(msg="Enter node(s) port number range end", mandatory=True)
            
            # FIXME: Do we have to connect() to the IP address/port# or just bind() to it?
            #
            
            is_ports_valid = True
            for port in range(configurables.node_port_range_low, configurables.node_port_range_high + 1):
                nodeServer = None
                try:
                    nodeServer = NodeServer(configurables.node_ip_address, port)
                except BaseException as be:
                    is_ports_valid = False
                
                
                # TODO: If we wish to connect() to the port then un-comment the following code.
                #
#                if (nodeServer != None):
#                    nodeServer.start()
#                    conn = socket.create_connection((configurables.node_ip_address, port), timeout=2)
#                    nodeServer.join()
            
            is_valid = is_ports_valid
        
        # Windows service?
        #
        got_YN = False
        while (got_YN == False):
            windows_service_yn = Console.getString(msg="Install node as a Windows service? (Y/N)", mandatory=True)
            if (windows_service_yn.upper() in "YN"):
                got_YN = True
                configurables.is_service = (windows_service_yn.upper() == "Y")
        

class DataStore(object):
    """The purpose of this class is to control the creation of the database.

    *Usage examples*::

        example code

    *Common Arguments for Class methods*:
        <arguments>
    
    *Audience*:
        <audience>
    
    *Constants*:
    """
    
    # Class constants
    #

    def __init__(self):
        # Initialise the Datastore class.
        #
        # Parameters:
        #     <param 1> :
        #     ...
        #
        
        self.logger = logging.getLogger('yarra.runtime.boot.DataStore')
    
    @staticmethod
    def exists(database_filename):
        """This function checks if the database exists.
        
        :param database_filename: String - pathname of database file.
        :rtype: True - database exists
        :rtype: False - database does not exist
        """
        
        return os.path.exists(database_filename)

    @staticmethod
    def delete(database_filename):
        """This function deletes the argument database filename.
        
        :param database_filename: String - pathname of database file.
        :rtype: None.
        """
        
        if (DataStore.exists(database_filename)):
            os.remove(database_filename)
    
    @staticmethod
    def write(database_filename, contents):
        """This function writes the argument contents to the argument
        file descriptor.
        
        :param database_filename: String - pathname of database file.
        :param contents: String - contents to write to datastore.
        :rtype: None.
        """

        file = open(database_filename, "wb+")
        file.write(contents)
        file.close()

class MetaLoader(object):
    """The purpose of this class is to play the role of the loader in the 
    Importer Protocol.

    *Usage examples*::

        example code

    *Common Arguments for Class methods*:
        <arguments>
    
    *Audience*:
        <audience>
    
    *Constants*:
    """
    
    # Class constants
    #

    def __init__(self, database_filename, resource_id, metadata_key):
        # Initialise the MetaLoader class.
        #
        # Parameters:
        #     database_filename: the file name of the datastore. 
        #     resouced_id: the id of the resouce that contains the module code. 
        #     metadata_key: the key to indicate whether the module path is a package or module.
        #
        
        self.logger = logging.getLogger('yarra.runtime.boot.MetaLoader')
        
        self._database_filename = database_filename
        self._resource_id = resource_id
        self._metadata_key = metadata_key
        
    def load_module(self, module_path):
        """This function searches the datastore for the module with the argument
        module path and returns the module.
        
        :param module_path: String - pathname of the module.
        :rtype: module.
        """

        # if module_path is a package
        #
        if (self._metadata_key == RESOURCE_KEY_URI_PACKAGE_PACKAGENAME):
            if not module_path in sys.modules:
                self.logger.log(logging.DEBUG, "Creating Module (%s)" % module_path)
                module = Install.create_python_module(module_path)
                
                # stuff you have to set for modules see PEP 302
                #
                module.__file__ = "<{0}>".format(self.__class__.__name__)
                module.__loader = self
                
                # stuff you have to set for packages see PEP 302
                #
                module.__path__ = [] 
            else:
                self.logger.log(logging.DEBUG, "Module (%s) already exists" % module_path)
            
        else:
            # module_path is a module
            #
            self.logger.log(logging.DEBUG, "Executing Code into (%s)" % module_path)       
            
            id = str(self._resource_id)
            conn = sqlite3.connect(self._database_filename)
            conn.row_factory = sqlite3.Row
            cursor = conn.cursor()          
            cursor.execute("""
                            SELECT r.id as {0}, r.object as {1}
                            FROM RESOURCE r
                            WHERE r.id = (?)
                            """.format(RESOURCE_ID, RESOURCE_OBJECT),
                            [id])
            
            # only one row should return
            #
            resource_row = cursor.fetchone()
    
            serialised_resource = resource_row[RESOURCE_OBJECT]
            module_code_encoded = Install.get_first_item_of_serialised_resource(serialised_resource)
            
            # we must decode the module code as it was encoded before being written to the datastore.
            #
            module_code = base64.b64decode(module_code_encoded)
            
            # reuse the same module if it is already there. See PEP 302 for explanation
            if (module_path in sys.modules):
                module = sys.modules[module_path]
            else:
                module = Install.create_python_module(module_path)
                
                # stuff you have to set for modules see PEP 302
                #
                module.__file__ = "<{0}>".format(self.__class__.__name__)
                module.__loader = self 
                
            # execute module's code in the module's global name space
            #
            exec module_code in module.__dict__
        
            self.logger.log(logging.DEBUG, "Module (%s) loaded" % module_path)
            conn.close()                
                
        return sys.modules[module_path]
    
    def __cmp__(self, other):
        if (self._resource_id == other._resource_id) and (self._metadata_key == other._metadata_key):
            return 0
        else:
            return -1
    
class MetaFinder(object):
    """The purpose of this class is to play the role of the finder in the 
    Importer Protocol.

    *Usage examples*::

        example code

    *Common Arguments for Class methods*:
        <arguments>
    
    *Audience*:
        <audience>
    
    *Constants*:
    """
    
    # Class constants
    #
    
    def __init__(self, database_filename):
        # Initialise the MetaLoader class.
        #
        # Parameters:
        #     database_filename: the file name of the datastore. 
        #
        
        self.logger = logging.getLogger('yarra.runtime.boot.MetaFinder')
        self._database_filename = database_filename
    
    def find_module(self, fullname, path=None):
        """This function returns a MetaLoader instance if the module is found
        in the datastore, otherwise None. When None is returned, the import is
        of the module is delegated to the built in import function.
        
        :param fullname: String - full name of the module to import from the
        datastore.
        :rtype: MetaLoader.
        """
        
        self.logger.log(logging.DEBUG, "Looking for (%s)..." % fullname)
        
        # log.debug("find_module(): Database filename (%s)" % self._database_filename)
        # log.debug("find_module(): key (%s), value (%s)" % (RESOURCE_KEY_URI_PACKAGE_MODULENAME, fullname))
        # log.debug("find_module(): key (%s), value (%s)" % (RESOURCE_KEY_URI_PACKAGE_PACKAGENAME, fullname))
        
        try:
            conn = sqlite3.connect(self._database_filename)
            conn.row_factory = sqlite3.Row
            cursor = conn.cursor()  
            cursor.execute("""
                            SELECT resource_id as {0}, key as {1}
                            FROM RESOURCEMETADATA
                            WHERE 
                            (key = (?) AND value = (?))
                            OR
                            (key = (?) AND value = (?))
                            """.format(RESOURCE_ID, RESOURCE_METADATA_KEY),
                            [RESOURCE_KEY_URI_PACKAGE_MODULENAME,  
                             fullname,
                             RESOURCE_KEY_URI_PACKAGE_PACKAGENAME, 
                             fullname])
            row = cursor.fetchone()
            conn.close()   
            if (row):
                self.logger.log(logging.DEBUG, "Found (%s). Now loading from DB" % fullname)
                return MetaLoader(self._database_filename, row[RESOURCE_ID], row[RESOURCE_METADATA_KEY])
            else:
                self.logger.log(logging.WARNING, "Module (%s) not found in datastore." % fullname)
                return None
        except Exception:
            self.logger.exception("Failure retrieving module (%s) from datastore (%s)" % (fullname, self._database_filename))

class XMLFormatException(Exception):
    """This class represents an exception for bad XML.

    *Usage examples*::

        example code

    *Common Arguments for Class methods*:
        <arguments>
    
    *Audience*:
        <audience>
    
    *Constants*:
    """
    
    pass


def configure_python_import(distribution, disable_import_hookin=False):
    """This function configures Python to import modules from the 
    datastore. By default we want to load imported modules from the 
    datastore, however the caller can disable this by specifying 
    the argument 'disable_import_hookin' = True.
    
    :param distribution: Distribution - distribution configuration.
    """

    if (disable_import_hookin is False):
        # register the import hook.
        #
        sys.meta_path = [MetaFinder(os.path.join(distribution.database_pathname, distribution.database_filename))]


def keyboard_interrupt_handler(signum, frame):
    """This function is called when the user interrupts the process with a Ctrl-C
    from the command line. 
    
    :param signum: int - the signal number.
    :param frame: Frame Object - the current stack frame.
    """
    
    logging.log(logging.ERROR, "Signal (%d) caught. Exiting." % signum)
    sys.exit(0)


if __name__ == '__main__':
    
    database_filename = ""
    database_pathname = ""
    distribution = Distribution()
    
    # Remember the executable name for when we write to Windows Service.
    #
    distribution.exe_pathname = sys.executable
    
    # We infer the path to the datastore from the path of the executable.
    #
    distribution.database_pathname = os.path.dirname(distribution.exe_pathname)
    
    # ----- Signal Handlers -----
    #
    # Note: Apparently a problem with interrupts in a multi-threaded Python program
    #       is that an interrupt gets caught by a thread that ignores it. Hence the
    #       interrupt gets lost. The solution is to to run a root process that 
    #       only waits for interrupt signals. At startup this process should fork()
    #       itself, the new process doing all the work.
    #
    #       So the following signal handler should be uncommented when this module
    #       is written with os.fork().
    #
#    signal.signal(signal.SIGINT, keyboard_interrupt_handler)
    
    # ----- Logging -----
    #
    
    # Add new logging levels.
    #
    logging.addLevelName(LoggingLevels.DEVEL, 'DEVEL')
    logging.addLevelName(LoggingLevels.TRACE, 'TRACE')
    
    # Set output logging level for all loggers.
    #
#    logging.root.setLevel(LoggingLevels.TRACE)
    logging.root.setLevel(logging.DEBUG)
    
    # Set the format of all log messages.
    #
    streamHandler = logging.StreamHandler()
    formatter = logging.Formatter("[%(asctime)s %(levelname)-8s %(threadName)s %(name)s %(module)s:%(lineno)d::%(funcName)s] %(message)s")
    streamHandler.setFormatter(formatter)
    logging.root.addHandler(streamHandler)
    
    # TODO: Re-instate Windows Logging.
    #
    
    # Also send logging to Windows event logger.
    #
    # NOTE: The Windows event log handler has default knowledge of the 'logging' 
    #       module's log levels and how they relate to Windows Event Log levels.
    #       We must add knowledge for our own levels.
    #
#    ntEventLogHandler = NTEventLogHandler("HAF eDA")
#    ntEventLogHandler.setFormatter(formatter)
#    ntEventLogHandler.typemap = {
#        logging.DEBUG   : win32evtlog.EVENTLOG_INFORMATION_TYPE,
#        logging.INFO    : win32evtlog.EVENTLOG_INFORMATION_TYPE,
#        logging.WARN    : win32evtlog.EVENTLOG_WARNING_TYPE,
#        logging.ERROR   : win32evtlog.EVENTLOG_ERROR_TYPE,
#        logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE,
#        LoggingLevels.DEVEL : win32evtlog.EVENTLOG_INFORMATION_TYPE,
#        LoggingLevels.TRACE : win32evtlog.EVENTLOG_INFORMATION_TYPE
#        }
#    logging.root.addHandler(ntEventLogHandler)
    
    # Get logger for this module.
    #
    logger = logging.getLogger('yarra.runtime.boot')
    
    # ----- Command line -----
    #
    
    # Notes about running the node. There are two ways to run the node - either 
    # as a service or from a shell. When run as a service the node must register 
    # with Windows Services so Windows knows the node has started. When running
    # from a shell we don't need to register with Windows. Therefore we need 
    # something on the command line that tells us which startup we're performing.
    # Running as a service is the most likely scenario so we'll require the 
    # "--start" option when being started from a shell.
    #
    
    # Possible command lines
    # ----------------------
    #
    # To initialise the node (first run):
    #     <path>/boot.exe --initialise
    #
    # To install the node as a service:
    #     <path>/boot.exe --install <datastore name>
    #
    # To remove the node from Windows Services:
    #     <path>/boot.exe --remove <datastore name>
    #
    # To run the node as a service is:
    #     <path>/boot.exe <datastore name>
    #
    # To run the node from a shell is:
    #     <path>/boot.exe --start <datastore name>
    #
    
    # get command line arguments.
    #
    parser = OptionParser(usage=u"%s [<options>] <datastore file>" % sys.argv[0])
    parser.add_option("--initialise", dest=u"do_initialise", action="store_true", default=False, help=u"Initialise the node from scratch.")
    parser.add_option("--install", dest=u"do_install", action="store_true", default=False, help=u"Install the node as a Windows Service.")
    parser.add_option("--remove", dest=u"do_remove", action="store_true", default=False, help=u"Remove the node from Windows Services.")
    parser.add_option("--start", dest=u"do_shell_start", action="store_true", default=False, help=u"Start the node from a command-line shell.")
    parser.add_option("--dih", dest=u"disable_import_hookin", action="store_true", default=False, help=u"Disable Hookin - Disable Import Hookin and load python modules from the python path (localy) not from the datastore")
    (options, args) = parser.parse_args()
    options.script = os.path.split(sys.argv[0])[1]

    # we allow only one option.
    # - "--dih" can be specified with any other option, so don't check for it.
    #
    numOptions = 0
    if (options.do_initialise is True): numOptions += 1
    if (options.do_install is True): numOptions += 1
    if (options.do_remove is True): numOptions += 1
    if (options.do_shell_start is True): numOptions += 1
    if ((numOptions == 0) and (len(args) == 0)) or (numOptions > 1):
        logger.log(logging.ERROR, "Incorrect number of arguments")
        parser.error("Incorrect number of arguments")
    
    options.do_service_start = (numOptions == 0)
    
    if (len(args) > 0):
        database_filename = args[0]
        database_pathname = os.path.join(distribution.database_pathname, database_filename)
    
    logger.log(logging.DEBUG, "Datastore path (%s)" % database_pathname)
    
    if (options.do_install is True) or \
       (options.do_remove is True) or \
       (options.do_shell_start is True) or \
       (options.do_service_start is True):
        if (len(database_pathname) == 0):
            logger.log(logging.ERROR, "Must specify a datastore file name.")
            parser.error("Must specify a datastore file name.")
        elif (DataStore.exists(database_pathname) is not True):
            logger.log(logging.ERROR, "Cannot find datastore (%s)" % database_pathname)
            parser.error("Cannot find datastore (%s)" % database_pathname)

    # Note: If the user specifies "--initialise" this overrides the specification
    #       of the database - if the database exists it is deleted.
    #       
    if (options.do_initialise is True):
        logger.log(logging.INFO, "Initialising new node...") 
        
        if (DataStore.exists(database_pathname)):
            logger.log(logging.INFO, "Deleting database %s." % database_pathname)
            DataStore.delete(database_pathname)
        
        configurables = Configurables()
        
        # Generate new node_ID
        #
        logger.log(logging.INFO, "Generating new Node ID...") 
        Install.generate_node_ID(distribution)
        
        logger.log(logging.INFO, "Get node configuration from user...") 
        Install.get_network_configurables(configurables, distribution)
        
        # Download the distribution database...
        #
        logger.log(logging.INFO, "Installing distribution...") 

        # Write distribution datastore to disk with filename:
        # - <node-ID>.datastore
        #
        distribution.database_filename = str(distribution.node_ID) + ".datastore"
        DataStore.write(os.path.join(distribution.database_pathname, distribution.database_filename), distribution.distribution_datastore_contents)
        
        # Load runtime for writing configuration data to datastore and installing service...
        #
        configure_python_import(distribution, options.disable_import_hookin)
        from runtime.rt import Runtime
        rt = Runtime()
        rt.install(configurables, distribution)
        
        # Install Windows Service (but don't run it!)
        #
        if (configurables.is_service == True):
            logger.log(logging.INFO, "Installing node as a Windows Service...")
            
            # TODO: Handle services for OSX and Linux.
            #
            
            rt.installService(distribution)
    
    elif (options.do_install is True):
        logger.log(logging.INFO, "Installing node as a Windows Service...") 
        
        distribution.database_filename = database_filename
        
        # Load runtime for installing service...
        #
        configure_python_import(distribution, options.disable_import_hookin)
        from runtime.rt import Runtime
        Runtime().installService(distribution)

    elif (options.do_remove is True):
        logger.log(logging.INFO, "Removing node from Windows Services...") 
        
        distribution.database_filename = database_filename
        
        # Load runtime for removing service...
        #
        configure_python_import(distribution, options.disable_import_hookin)
        from runtime.rt import Runtime
        Runtime().removeService()

    elif (options.do_shell_start is True):
        logger.log(logging.INFO, "Starting the yarra environment...")
        
        distribution.database_filename = database_filename
        
        # We have been started from a shell...
        #
        
        # Load runtime from database and hand-off control.
        #
        # Now we start the yarra environment by calling rt.Runtime::start(). For 
        # the import statements in rt.py to work correctly we must load the yarra 
        # modules from the database, not the file system. Python allows us to do
        # this by registering a finder object (one with a 'find_module()' function) 
        # with 'sys.meta_path'. The finder object will search for the module in
        # the database and, if found, load the module into the Python runtime.
        #
        # For a full explanation see PEP 302:
        # - http://www.python.org/dev/peps/pep-0302/
        # 
        
        # Load runtime for starting process...
        #
        configure_python_import(distribution, options.disable_import_hookin)
        from runtime.rt import Runtime
        Runtime().startProcess(distribution)
        
    elif (options.do_service_start is True):
        logger.log(logging.INFO, "Starting the yarra environment...")
        
        distribution.database_filename = database_filename
        
        # Load runtime for starting service...
        #
        configure_python_import(distribution, options.disable_import_hookin)
        from runtime.rt import Runtime
        Runtime().startAsService(distribution)
    
    sys.exit()